﻿using System;
using System.Collections.Generic;
using System.Data;
using System.IO;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;

namespace QT.SEO
{
    /// <summary>
    /// URL采集的类
    /// </summary>
    public class UrlGather
    {
        /// <summary>
        /// 通过url对应获取title、快照时间、快照地址等
        /// </summary>
        /// <param name="HtmlData">网页源代码</param>
        /// <param name="dr">搜索引擎信息</param>
        /// <param name="url">用于匹配的url地址，用来替换正则表达式中的相关参数</param>
        /// <param name="encoding">搜索引擎编码</param>
        /// <returns>SearchResult</returns>
        public static SearchResult GetSnapShot(string HtmlData, DataRow dr, string url, Encoding encoding)
        {
            SearchResult sr = new SearchResult();
            sr.url = url;
            sr.IsSnap = false;

            #region 获取结果区域

            Regex myResultRegex = new Regex(dr["ResultRegex"].ToString(), RegexOptions.IgnoreCase);
            Match ResultMatch = myResultRegex.Match(HtmlData);
            if (ResultMatch.Length > 0)
            {
                #region 获取分段区域

                Regex mySectionRegex = new Regex(dr["SectionRegex"].ToString(), RegexOptions.IgnoreCase);
                MatchCollection mc = mySectionRegex.Matches(ResultMatch.Value);
                if (mc.Count > 0)
                {
                    foreach (Match mt in mc)
                    {
                        #region 获取页面信息区域

                        Regex myHtmlRegex = new Regex(dr["HtmlRegex"].ToString().ToLower().Replace("{url}", url), RegexOptions.IgnoreCase);
                        Match HtmlMatch = myHtmlRegex.Match(mt.Value);

                        if (HtmlMatch.Length > 0)
                        {
                            #region 获取页面信息

                            string DefaultPages = QT.SEO.Setting.Get("DefaultPage");
                            Regex cpRegex = new Regex(("^(http://)?{url}(/)?(" + DefaultPages + ")?$").Replace(".", @"\.").Replace("{url}", url), RegexOptions.IgnoreCase);
                            bool b = false;

                            b = cpRegex.IsMatch(HtmlMatch.Groups["url"].Value);
                            if (!b) b = GetURL(HtmlMatch.Groups["url"].Value, "", encoding, new Regex(("(http://)?{url}(/)?(" + DefaultPages + ")?").Replace(".", @"\.").Replace("{url}", url), RegexOptions.IgnoreCase));
                            if (!b) b = cpRegex.IsMatch(CheckUrl(HtmlMatch.Groups["url"].Value));
                            if (!b)
                            {
                                sr.IsSnap = false;
                                string myData = GetHtml(sr.url, "", Encoding.Default);
                                Regex myRegex = new Regex("<title>(?<title>(.*?))</title>", RegexOptions.IgnoreCase);
                                sr.title = myRegex.Match(myData).Groups["title"].Value;
                                if (sr.title == null) sr.title = "";
                                continue;
                            }

                            Regex regex = new Regex(@"<(.*?)>");
                            sr.title = regex.Replace(HtmlMatch.Groups["title"].Value, "");
                            sr.IsSnap = true;

                            #endregion

                            #region 获取快照（区域）信息

                            Regex mySnapRegex = new Regex(dr["SnapRegex"].ToString(), RegexOptions.IgnoreCase);
                            Match SnapMatch = mySnapRegex.Match(mt.Value);

                            if (SnapMatch.Length > 0)
                            {
                                sr.date = SnapMatch.Groups["date"].Value;
                                sr.snapshot = SnapMatch.Groups["snapshot"].Value;
                            }
                            else
                            {
                                if (sr.IsSnap) sr.date = "无快照";
                            }

                            break;

                            #endregion
                        }

                        #endregion
                    }
                }

                #endregion
            }

            #endregion

            if (sr.IsSnap == false)
            {
                string myData = GetHtml(sr.url, "", Encoding.Default);
                Regex myRegex = new Regex("<title>(?<title>(.*?))</title>", RegexOptions.IgnoreCase);
                sr.title = myRegex.Match(myData).Groups["title"].Value;
                if (sr.title == null) sr.title = "";
                sr.date = "未收录";
                sr.snapshot = "";
            }

            return sr;
        }

        /// <summary>
        /// 通过url对应获取排名、页码
        /// </summary>
        /// <param name="HtmlData">网页源代码</param>
        /// <param name="dr">搜索引擎信息</param>
        /// <returns>int</returns>
        public static KeywordResult GetKeyword(string HtmlData, DataRow dr, string url, Encoding encoding)
        { 
            KeywordResult kr = new KeywordResult();
            kr.url = url;
            kr.id="0";
            kr.page="0";

            #region 获取结果区域

            Regex myResultRegex = new Regex(dr["ResultRegex"].ToString(), RegexOptions.IgnoreCase);
            Match ResultMatch = myResultRegex.Match(HtmlData);
            if (ResultMatch.Length > 0)
            {
                #region 获取分段区域

                Regex mySectionRegex = new Regex(dr["SectionRegex"].ToString(), RegexOptions.IgnoreCase);
                MatchCollection mc = mySectionRegex.Matches(ResultMatch.Value);
                if (mc.Count > 0)
                {
                    foreach (Match mt in mc)
                    {
                        #region 获取页面信息区域

                        Regex myHtmlRegex = new Regex(dr["HtmlRegex"].ToString().ToLower().Replace("{url}", url), RegexOptions.IgnoreCase);
                        Match HtmlMatch = myHtmlRegex.Match(mt.Value);

                        if (HtmlMatch.Length > 0)
                        {
                            #region 获取排名信息

                            string DefaultPages = QT.SEO.Setting.Get("DefaultPage");
                            Regex cpRegex = new Regex(("^(http://)?{url}(/)?(" + DefaultPages + ")?$").Replace(".", @"\.").Replace("{url}", url), RegexOptions.IgnoreCase);
                            bool b = false;

                            b = cpRegex.IsMatch(HtmlMatch.Groups["url"].Value);
                            if (!b) b = GetURL(HtmlMatch.Groups["url"].Value, "", encoding, new Regex(("(http://)?{url}(/)?(" + DefaultPages + ")?").Replace(".", @"\.").Replace("{url}", url), RegexOptions.IgnoreCase));
                            if (!b) b = cpRegex.IsMatch(CheckUrl(HtmlMatch.Groups["url"].Value));
                            if (!b) continue;

                            kr.id = HtmlMatch.Groups["id"].Value;

                            #region 获取快照（区域）信息

                            Regex mySnapRegex = new Regex(dr["SnapRegex"].ToString(), RegexOptions.IgnoreCase);
                            Match SnapMatch = mySnapRegex.Match(mt.Value);

                            if (SnapMatch.Length > 0)
                            {
                                kr.date = SnapMatch.Groups["date"].Value;
                            }
                            else
                            {
                                kr.date = "无快照";
                            }

                            #endregion

                            break;

                            #endregion
                        }

                        #endregion
                    }
                }

                #endregion
            }

            #endregion

            return kr;
        }

        /// <summary>
        /// 获取收录量
        /// </summary>
        /// <param name="HtmlData">网页源代码</param>
        /// <param name="dr">搜索引擎信息</param>
        /// <returns>long</returns>
        public static long GetCount(string HtmlData, DataRow dr)
        {
            long count=0;
            Regex myRCountRegex = new Regex(dr["CountRegex"].ToString(), RegexOptions.IgnoreCase);
            Match mt = myRCountRegex.Match(HtmlData);
            if (mt.Length > 0) count = int.Parse(mt.Groups["count"].Value.Replace(",", ""));

            return count;
        }

        /// <summary>
        /// 获取最终访问网页的地址
        /// </summary>
        /// <param name="url">初始访问的页面地址</param>
        /// <returns>string</returns>
        public static string CheckUrl(string url)
        {
            string outURL = "";
            HttpWebRequest httpWebRequest = null;
            HttpWebResponse webResponse = null;
            try
            {
                httpWebRequest = (HttpWebRequest)HttpWebRequest.Create(url.ToLower().StartsWith("http://") ? url : ("http://" + url));
                httpWebRequest.Referer = "";
                httpWebRequest.ContentType = "application/x-www-form-urlencoded";
                httpWebRequest.Accept = "image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/x-shockwave-flash, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*";
                httpWebRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727,; .NET CLR 3.0.04506,; .NET CLR 3.5.21022)";
                httpWebRequest.Method = "GET";
                webResponse = (HttpWebResponse)httpWebRequest.GetResponse();
                outURL = webResponse.ResponseUri.ToString();
            }
            catch
            {
            }
            finally
            {
                if (webResponse != null) webResponse.Close();
            }
            return outURL;
        }

        /// <summary>
        /// 获取访问网页的返回状态
        /// </summary>
        /// <param name="url">初始访问的页面地址</param>
        /// <returns>HttpStatusCode</returns>
        public static HttpStatusCode GetStatusCode(string url)
        {
            HttpStatusCode status = HttpStatusCode.NotFound;
            HttpWebRequest httpWebRequest = null;
            HttpWebResponse webResponse = null;
            try
            {
                httpWebRequest = (HttpWebRequest)HttpWebRequest.Create(url.ToLower().StartsWith("http://") ? url : ("http://" + url));
                httpWebRequest.Referer = "";
                httpWebRequest.ContentType = "application/x-www-form-urlencoded";
                httpWebRequest.Accept = "image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/x-shockwave-flash, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*";
                httpWebRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727,; .NET CLR 3.0.04506,; .NET CLR 3.5.21022)";
                httpWebRequest.Method = "GET";
                webResponse = (HttpWebResponse)httpWebRequest.GetResponse();
                status = webResponse.StatusCode;
            }
            catch(WebException ex)
            {
                status = ((HttpWebResponse)ex.Response).StatusCode;
            }
            finally
            {
                if (webResponse != null) webResponse.Close();
            }
            return status;
        }

        /// <summary>
        /// 获取网页源代码
        /// </summary>
        /// <param name="url">要访问的网址</param>
        /// <param name="ref_url">来源网址</param>
        /// <param name="encode">编码</param>
        /// <returns>string</returns>
        public static string GetHtml(string url, string ref_url, Encoding encode)
        {
            string outHtmlData = "";
            HttpWebRequest httpWebRequest = null;
            HttpWebResponse webResponse = null;
            try
            {
                httpWebRequest = (HttpWebRequest)HttpWebRequest.Create(url.ToLower().StartsWith("http://") ? url : ("http://" + url));
                httpWebRequest.Referer =  ref_url;
                httpWebRequest.ContentType = "application/x-www-form-urlencoded";
                httpWebRequest.Accept = "image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/x-shockwave-flash, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*";
                httpWebRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727,; .NET CLR 3.0.04506,; .NET CLR 3.5.21022)";
                httpWebRequest.Method = "GET";
                webResponse = (HttpWebResponse)httpWebRequest.GetResponse();
                if (webResponse.StatusCode == HttpStatusCode.OK)
                {
                    StreamReader sr = new StreamReader(webResponse.GetResponseStream(), encode);
                    outHtmlData = sr.ReadToEnd();
                    sr.Close();
                }
                else
                {
                    outHtmlData = "<title>" + webResponse.StatusCode + "</title>";
                }
                webResponse.Close();
            }
            catch(WebException ex)
            {
                return "<title>" + ((HttpWebResponse)ex.Response).StatusCode + "</title>";
            }
            catch
            {
                return "";
            }
            return outHtmlData;
        }
        
        /// <summary>
        /// 获取302跳转页面的URL
        /// </summary>
        /// <param name="url">要访问的网址</param>
        /// <param name="ref_url">来源网址</param>
        /// <param name="encode">编码</param>
        /// <param name="regex">正则表达式，用来匹配302跳转的网址</param>
        /// <returns>bool</returns>
        public static bool GetURL(string url, string ref_url, Encoding encode, Regex regex)
        {
            HttpWebRequest httpWebRequest = null;
            HttpWebResponse webResponse = null;
            bool b = false;
            try
            {
                httpWebRequest = (HttpWebRequest)HttpWebRequest.Create(url.ToLower().StartsWith("http://") ? url : ("http://" + url));
                httpWebRequest.AllowAutoRedirect = false;
                httpWebRequest.Referer = ref_url;
                httpWebRequest.ContentType = "application/x-www-form-urlencoded";
                httpWebRequest.Accept = "image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/x-shockwave-flash, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*";
                httpWebRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727,; .NET CLR 3.0.04506,; .NET CLR 3.5.21022)";
                httpWebRequest.Method = "GET";
                webResponse = (HttpWebResponse)httpWebRequest.GetResponse();
                StreamReader sr = new StreamReader(webResponse.GetResponseStream(), encode);
                b = regex.IsMatch(sr.ReadToEnd());

                webResponse.Close();
            }
            catch
            {                
            }

            return b;
        }
    }

    /// <summary>
    /// 快照搜索使用的Struct
    /// </summary>
    public struct SearchResult
    {
        /// <summary>
        /// GUID
        /// </summary>
        public string guid;
        /// <summary>
        /// 网址
        /// </summary>
        public string url;
        /// <summary>
        /// 网页标题
        /// </summary>
        public string title;
        /// <summary>
        /// 快照日期
        /// </summary>
        public string date;
        /// <summary>
        /// 快照地址
        /// </summary>
        public string snapshot;
        /// <summary>
        /// 是否存在快照
        /// </summary>
        public bool IsSnap;
    }

    /// <summary>
    /// 关键字搜索使用的Struct
    /// </summary>
    public struct KeywordResult
    {
        /// <summary>
        /// GUID
        /// </summary>
        public string guid;
        /// <summary>
        /// 网址
        /// </summary>
        public string url;
        /// <summary>
        /// 关键字
        /// </summary>
        public string keyword;
        /// <summary>
        /// 搜索结果排名
        /// </summary>
        public string id;
        /// <summary>
        /// 搜索结果页码
        /// </summary>
        public string page;
        /// <summary>
        /// 快照日期
        /// </summary>
        public string date;
        /// <summary>
        /// 结果网页地址
        /// </summary>
        public string ResultUrl;
    }
}
